#!/usr/bin/perl

=head1 NAME

    referred-iterator.pl - Collect referred_by fields from a set of files and look for matches in same dir

=head1 SYNOPSIS

    Use with Poly Speed Dating file formats.  Looks at .txt files in current working directory
    and outputs to referred-matches.csv

    =head1 DESCRIPTION

    A collection of text files each have a referred_by field, e.g:

referred_by: Michael Baum

    They also have names and whatnot in them.  They also have an ID:

ID: LM394

    This script produces a csv file of the form:

    ID, referred_by, ID_Match_List
    LM394, "Michael Baum", MB303 MB202 MB504

    generated by doing a grep of the individual words in referred_by in
    all the other text files, and pulling the IDs of all those files.  One
    easy-making thing is the files have the IDs as their first part of the
    name (so if you did a grep and got filenames, the first chunk of the
    filename would be the list of matches.

=head1 AUTHOR

    Ert Dredge <ert@alum.mit.edu>

=cut



use strict;
use warnings;

my $mydirectory = "run5";

# File where our newly generated CSV goes
my $outputfilename = "referred-matches.csv";


# Iterate over every file in the specified directory,
# collecting the names of *.txt files
opendir (THISDIR, $mydirectory) or die "$!";
my @files = grep {/.*\.txt/} readdir THISDIR;
close THISDIR;

# Open the output
open (OUTFILE, ">$outputfilename");

# Make word kill list
my @killlist = ('email', 'Chasing', 'Social', 'Club', 'Amy', 'BABN', 'north', 'bay', 'fetlife', 'fetlife.com', 'coffee', 'Peppermint', 'yahoo', 'group', 'EBPP', 'pepper,', 'sfbaypoly', 'tribe', 'last', 'year', 'SFPoly.org', 'Friend', 'tribe.net', 'on', 'community', 'luke', 'I', 'have', 'post', 'pepper', 'mint', 'Mint', 'too', 'at', 'conference', 'Pepper', 'in', 'and', 'the', 'site', 'other', 'SF', 'poly', 'meetup', 'Livejournal', 'My', 'landlord', 'boyfriend', 'one', 'of', 'online', 'newletters', 'eastbay', 'polypotluck', 'list' );
my %killset = map{ (lc $_) => 1 } @killlist;
print "\n\n\nKILL LIST: " .  join( " ", keys (%killset) ) . "\n";


# Iterate over each of the .txt files we collected
foreach my $thisfilename (@files) {
    # Open the file
    open (THISFILE, "$mydirectory/$thisfilename") or die "$!" . "\nFilename: " . $thisfilename;

    # Iterate over each line in the file
    while (<THISFILE>) {
        if (( m/^referred_by: (.*)/) ||( m/^remarks: (.*)/ )) {
            # Found a referred-by field.  The matching will have left
            # the value of the field in $1

            # Trim off newline character if present
            my $referred_by = $1;
            chomp $referred_by;
            
            print "Examining $thisfilename: $referred_by\n";

            if ($referred_by) {
                # They entered something for referred by.

                my @allmatches = ();
                my $finalmatches = "";

                # First try the whole phrase
                if ( $killset{ lc $referred_by } ) {
                    print "Killing entire referred by\n";
                } else {
                    my @thismatches = `grep -l -i '$referred_by' $mydirectory/*.txt`;
                    chomp @thismatches;

                    # We always match ourself -- see if we match more than one
                    if ($#thismatches > 1) {
                        # Got a complete match on someone else
                        push @allmatches, @thismatches;

                    } else {

                        # Nope, we just matched ourself.
                        # Try each individual word
                        my @words = split (/\W+/, $referred_by);

                        foreach my $thisword (@words) {

                            if ( $killset{ lc $thisword } || (length($thisword)<4) ) {
                                print "     Killing $thisword\n";
                            } else {
                                print "    saving |$thisword|\n";

                                # Collect in @thesematches array
                                my @thesematches = `grep -w -l -i $thisword $mydirectory/*.txt`;
                                chomp @thesematches;
                                if ( $#thesematches < 4 ) {
                                    push @allmatches, @thesematches;
                                }
                            }

                        } # foreach my $thisword (@words)
                        
                    } #  if ($#thismatches > 1) ... else ...

                    # Remove duplicates
                    my %uniquematches = map { $_ => 1 } @allmatches;
                    
                    # Remove match to self
                    print "o$thisfilename|\n";

                    delete $uniquematches{"$mydirectory/$thisfilename"};
                    
                    # Collect in single string
                    $finalmatches = join (" ", keys (%uniquematches));
                    
                    # Remove the .txt extenders
                    (my $thisid = $thisfilename) =~ s/.txt$//g;
                    (my $matchedids = $finalmatches) =~ s/.txt//g;
                    ($matchedids = $matchedids) =~ s/$mydirectory\///g;

                    # Output
#	      if ($matchedids) {
#              ($thisid = $thisid) ~= s/_\d+//g;
                    print OUTFILE "$thisid,\"$referred_by\",$matchedids\n";
#	      }
                }        
            } #  if ($referred_by)
            
        } # if (m/^referred_by: (.*)/)
        
    } # while (<THISFILE>)

    close THISFILE;

} # foreach my $thisfilename (@files)

close OUTFILE;
